{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25426f26",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import math\n",
    "from tqdm import tqdm\n",
    "import re\n",
    "from collections import Counter\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dadcaecb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0fff3ea9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 4, 0, 9, 3]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random.sample(range(10), 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "78fad71b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchtext.data.utils import get_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "18d80b42",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = get_tokenizer(\"basic_english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c860518f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i', 'like', 'apple', ',', 'what', 'do', 'you', 'like']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer('i like apple, what do you like')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "86d4f472",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['like']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer('LIKE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4f364442",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[13, 12, 8]\n"
     ]
    }
   ],
   "source": [
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "from torchtext.data.utils import get_tokenizer\n",
    "# 定义分词器\n",
    "tokenizer = get_tokenizer(\"basic_english\")\n",
    "# 模拟数据迭代器\n",
    "def yield_tokens(data_iter):\n",
    "   for text in data_iter:\n",
    "       yield tokenizer(text)\n",
    "# 示例数据\n",
    "data = [\"This is a test\", \"Another example of text data\"]\n",
    "# 构建词汇表\n",
    "vocab = build_vocab_from_iterator(yield_tokens(data), specials=[\"<PAD>\", \"<UNK>\", \"<SOS>\", \"<EOS>\", \"<SEP>\"])\n",
    "vocab.set_default_index(vocab[\"<UNK>\"])\n",
    "# 测试词汇表\n",
    "print(vocab([\"this\", \"text\", \"example\"])) # 输出：[4, 5, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f10404c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[13, 9, 5]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab(['this','is','a'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0ce20316",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'text': 12,\n",
       " '<EOS>': 3,\n",
       " '<PAD>': 0,\n",
       " '<SEP>': 4,\n",
       " 'this': 13,\n",
       " 'example': 8,\n",
       " '<UNK>': 1,\n",
       " 'another': 6,\n",
       " 'a': 5,\n",
       " '<SOS>': 2,\n",
       " 'data': 7,\n",
       " 'is': 9,\n",
       " 'of': 10,\n",
       " 'test': 11}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab.get_stoi()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "360e4f3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def out(data):\n",
    "    for _ in data:\n",
    "        yield _\n",
    "\n",
    "v=build_vocab_from_iterator(out([['I','like','you']]), specials=[\"<unk>\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "067235f5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v(['I'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "51e825ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[2]"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v(['like'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "670e2f2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['ilik', 'eyou']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s=\"ilik eyou\"\n",
    "s.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "9c75b022",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i', 'like', 'you']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer(\"i like you\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9ac85076",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "目标长度： 5\n",
      "词汇表索引对应：\n",
      "['<PAD>', '<UNK>', '<SOS>', '<EOS>', '<SEP>', 'Hello', 'I', 'NLP', 'This', 'a', 'is', 'like', 'long', 'sentence', 'world']\n",
      "\n",
      "填充后的序列：\n",
      "[6, 11, 7, 0, 0]\n",
      "[5, 14, 0, 0, 0]\n",
      "[8, 10, 9, 12, 13]\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torchtext.vocab import build_vocab_from_iterator\n",
    "\n",
    "# 1. 准备示例数据（不同长度的句子）\n",
    "sentences = [\n",
    "    [\"I\", \"like\", \"NLP\"],\n",
    "    [\"Hello\", \"world\"],\n",
    "    [\"This\", \"is\", \"a\", \"long\", \"sentence\"]\n",
    "]\n",
    "\n",
    "# 2. 构建包含 <PAD> 等特殊符号的词汇表\n",
    "specials = [\"<PAD>\", \"<UNK>\", \"<SOS>\", \"<EOS>\", \"<SEP>\"]\n",
    "vocab = build_vocab_from_iterator(sentences, specials=specials)\n",
    "\n",
    "# 设置 <UNK> 为默认索引（处理未登录词）\n",
    "vocab.set_default_index(vocab[\"<UNK>\"])\n",
    "\n",
    "# 3. 确定填充后的固定长度（取最长句子长度，或自定义）\n",
    "max_length = max(len(sent) for sent in sentences)  # 这里最长句子长度为 5\n",
    "print(\"目标长度：\", max_length)  # 输出：5\n",
    "\n",
    "# 4. 将句子转换为索引，并使用 <PAD> 填充\n",
    "padded_sequences = []\n",
    "for sent in sentences:\n",
    "    # 句子转换为索引\n",
    "    indices = [vocab[word] for word in sent]\n",
    "    # 计算需要填充的长度\n",
    "    pad_length = max_length - len(indices)\n",
    "    # 用 <PAD> 的索引填充（vocab[\"<PAD>\"] 通常为 0）\n",
    "    padded = indices + [vocab[\"<PAD>\"]] * pad_length\n",
    "    padded_sequences.append(padded)\n",
    "\n",
    "# 5. 查看结果\n",
    "print(\"词汇表索引对应：\")\n",
    "print(vocab.get_itos())  # 前5个是特殊符号：['<PAD>', '<UNK>', '<SOS>', '<EOS>', '<SEP>', ...]\n",
    "\n",
    "print(\"\\n填充后的序列：\")\n",
    "for seq in padded_sequences:\n",
    "    print(seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f7ac0a14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[6, 11, 7, 0, 0], [5, 14, 0, 0, 0], [8, 10, 9, 12, 13]]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "padded_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "4b92aecc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ True,  True,  True],\n",
       "        [False,  True,  True],\n",
       "        [False, False,  True]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a=torch.triu(torch.ones(3, 3))==1\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d618971c",
   "metadata": {},
   "outputs": [],
   "source": [
    "a=a.transpose(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ac1030c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [1., 1., 0.],\n",
       "        [1., 1., 1.]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c4629631",
   "metadata": {},
   "outputs": [],
   "source": [
    "a=a.float().masked_fill(a==0, float('-inf'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "6a5075af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., -inf, -inf],\n",
       "        [0., 0., -inf],\n",
       "        [0., 0., 0.]])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.masked_fill(a == 1, float(0.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "93e7533d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "掩蔽矩阵形状: torch.Size([5, 5])\n",
      "掩蔽矩阵内容:\n",
      " tensor([[0., -inf, -inf, -inf, -inf],\n",
      "        [0., 0., -inf, -inf, -inf],\n",
      "        [0., 0., 0., -inf, -inf],\n",
      "        [0., 0., 0., 0., -inf],\n",
      "        [0., 0., 0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "# 生成一个5×5的掩蔽矩阵（假设序列长度为5）\n",
    "seq_len = 5\n",
    "mask = nn.Transformer.generate_square_subsequent_mask(seq_len)\n",
    "\n",
    "print(\"掩蔽矩阵形状:\", mask.shape)  # 输出: torch.Size([5, 5])\n",
    "print(\"掩蔽矩阵内容:\\n\", mask)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b2c70f17",
   "metadata": {},
   "outputs": [],
   "source": [
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bd0228e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "65e05ad0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22.627416997969522"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "math.sqrt(512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "fa6400a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "input2 = \"i like you\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a2ca3487",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "''"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input(\"i like you\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ff581cba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.1280, -0.9369,  1.4110,  0.4453,  1.7081,  0.7840, -0.2503,\n",
       "          -0.7136, -2.4942, -2.0621,  0.6288, -0.9204],\n",
       "         [-0.8393,  0.9111, -0.6746,  1.0287, -0.1100,  1.5170, -0.4055,\n",
       "           0.4011,  0.4714, -0.9290,  0.6046,  0.1102],\n",
       "         [-0.1540, -0.9379,  0.3355,  0.7879, -0.2415,  0.2540,  0.5555,\n",
       "           0.3444, -0.2124,  0.0681, -0.4312,  0.1042],\n",
       "         [-0.8158, -1.3356, -0.9504, -0.7224,  0.3239,  2.1119,  1.3106,\n",
       "           0.3865,  1.2064,  1.6655,  0.1090,  0.3254],\n",
       "         [ 0.4986, -0.8056, -0.9950, -0.4546,  0.0610, -1.3025,  1.2651,\n",
       "           1.4858,  0.7240,  1.9714, -0.4648,  1.4534]]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "a=torch.randn(1,5,12)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "902d74e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0478, 0.0213, 0.2229, 0.0849, 0.3000, 0.1191, 0.0423, 0.0266,\n",
       "          0.0045, 0.0069, 0.1020, 0.0217],\n",
       "         [0.0230, 0.1324, 0.0271, 0.1489, 0.0477, 0.2427, 0.0355, 0.0795,\n",
       "          0.0853, 0.0210, 0.0975, 0.0594],\n",
       "         [0.0625, 0.0285, 0.1020, 0.1603, 0.0573, 0.0940, 0.1271, 0.1029,\n",
       "          0.0590, 0.0781, 0.0474, 0.0809],\n",
       "         [0.0161, 0.0096, 0.0140, 0.0176, 0.0502, 0.3002, 0.1347, 0.0535,\n",
       "          0.1214, 0.1921, 0.0405, 0.0503],\n",
       "         [0.0620, 0.0168, 0.0139, 0.0239, 0.0400, 0.0102, 0.1335, 0.1665,\n",
       "          0.0777, 0.2705, 0.0237, 0.1611]]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b=torch.softmax(a,dim=-1)\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "13e69b8a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0478, 0.0213, 0.2229, 0.0849, 0.3000, 0.1191, 0.0423, 0.0266,\n",
       "          0.0045, 0.0069, 0.1020, 0.0217],\n",
       "         [0.0230, 0.1324, 0.0271, 0.1489, 0.0477, 0.2427, 0.0355, 0.0795,\n",
       "          0.0853, 0.0210, 0.0975, 0.0594],\n",
       "         [0.0625, 0.0285, 0.1020, 0.1603, 0.0573, 0.0940, 0.1271, 0.1029,\n",
       "          0.0590, 0.0781, 0.0474, 0.0809],\n",
       "         [0.0161, 0.0096, 0.0140, 0.0176, 0.0502, 0.3002, 0.1347, 0.0535,\n",
       "          0.1214, 0.1921, 0.0405, 0.0503],\n",
       "         [0.0620, 0.0168, 0.0139, 0.0239, 0.0400, 0.0102, 0.1335, 0.1665,\n",
       "          0.0777, 0.2705, 0.0237, 0.1611]]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7dc910ff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0620, 0.0168, 0.0139, 0.0239, 0.0400, 0.0102, 0.1335, 0.1665, 0.0777,\n",
       "        0.2705, 0.0237, 0.1611])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b=b[0, -1, :]\n",
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1f5965c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([9])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.multinomial(b, num_samples=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "098b5d74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[-0.1280, -0.9369,  1.4110,  0.4453,  1.7081,  0.7840, -0.2503,\n",
       "          -0.7136, -2.4942, -2.0621,  0.6288, -0.9204],\n",
       "         [-0.8393,  0.9111, -0.6746,  1.0287, -0.1100,  1.5170, -0.4055,\n",
       "           0.4011,  0.4714, -0.9290,  0.6046,  0.1102],\n",
       "         [-0.1540, -0.9379,  0.3355,  0.7879, -0.2415,  0.2540,  0.5555,\n",
       "           0.3444, -0.2124,  0.0681, -0.4312,  0.1042],\n",
       "         [-0.8158, -1.3356, -0.9504, -0.7224,  0.3239,  2.1119,  1.3106,\n",
       "           0.3865,  1.2064,  1.6655,  0.1090,  0.3254],\n",
       "         [ 0.4986, -0.8056, -0.9950, -0.4546,  0.0610, -1.3025,  1.2651,\n",
       "           1.4858,  0.7240,  1.9714, -0.4648,  1.4534]]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c214abe2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 10, 12])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat([a,a],dim=1).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "50ade62a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 5, 12])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "813726d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "s=torch.randn(1,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "04cec999",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.1975, -0.9246, -0.1102],\n",
       "        [ 0.1975, -0.9246, -0.1102]])"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.cat([s,s],dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a68603ab",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch_py38",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.7rc1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
